import scrapy
from scrapy import Selector, Request
from Spider.items import MovieItem
from scrapy.http import HtmlResponse


class DoubanSpider(scrapy.Spider):
    name = "douban"
    allowed_domains = ["movie.douban.com"]
    # start_urls = ["https://movie.douban.com/top250"]

    # 自定义url请求
    def start_requests(self):
        for page in range(10):
            yield scrapy.Request(
                url=f'https://movie.douban.com/top250?start={page * 25}&filter='
            )


    def parse(self, response: HtmlResponse, **kwargs):
        sel = Selector(response)
        list_items = sel.css('#content > div > div.article > ol > li')
        for i in list_items:
            movie_item = MovieItem()
            # extract_first() :抽取标签里面的第一条数据
            movie_url = i.css('div.info > div.hd > a::attr(href)').extract_first().strip()
            movie_item['title'] = i.css('span.title::text').extract_first().strip()
            movie_item['rank'] = i.css('span.rating_num::text').extract_first().strip()
            movie_item['subject'] = i.css('span.inq::text').extract_first() or ''
            # 生成器
            yield Request(
                url=movie_url,
                callback=self.parse_movie,
                cb_kwargs={
                    'item': movie_item
                }
            )

    def parse_movie(self, response, **kwargs):
        movie_item = kwargs['item']
        sel = Selector(response)
        movie_item['duration'] = sel.css('span[property="v:runtime"]::attr(content)').extract_first()
        movie_item['introduce'] = sel.css('span[property="v:summary"]::text').extract_first()
        yield movie_item


        # 拿到分页符里面的a标签信息
        # href_list = sel.css('#content > div > div.article > div.paginator > a::attr(href)')
        # for a in href_list:
        #     url = response.urljoin(a.extract())
        #     yield Request(url=url)